const requests = require('request')

// const cheerio = require('cheerio')
// const Axios = require('axios');
// const fs = require('fs')
const xpath = require('xpath')
const dom = require('xmldom-silent').DOMParser

const logger = require('./logs/log.js')
// const logger = log.logger
const util = require('./libs/utils.js')
const config = require('./libs/config.js')
const ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'

// async function parseUrl(url) {
//     const respose = await Axios({
//         method: 'get',
//         url: url,
//         proxy: false,
//         timeout: 120000,
//         headers: {
//           'user-agent': ua,
//         },
//       });
//     const resposeData = respose.data
//     return resposeData
// }

// function cheerioParseData(body) {
//     const $ = cheerio.load(body)
//     var anText = $('#antiforgery_token').text()
//     var anValue = $('#antiforgery_token').attr('value')
//     console.log(`anText: ${anText}, anValue: ${anValue}`)
// }
var proxys = [];

async function parseUrl(url) {
    var proxy = proxys[0];
    return new Promise((resolve, reject) => {
        requests.get({
            url: url,
            port: proxy.port,
            host: proxy.ip,
            headers: {
                'user-agent': ua
            }
        }, (error, response, body) => {
            if(!error && response.statusCode == 200) {
                resolve(body)
            }else {
                console.log(error.message)
                reject({})
            }
        })
    })
}

async function parseListData(html) {
    var datasList = [];
    var doc = new dom().parseFromString(html);
    var baseXpaths = '//*[@id="mainContent"]/div/div[@class="day"]';
    var titleXpath = 'string(./div[@class="postTitle"]/a/span/text())';
    var hrefXpath = 'string(./div[@class="postTitle"]/a/@href)';
    var items = xpath.select(baseXpaths, doc);
    items.forEach(function (item) {
        var datas = {};
        var title = xpath.select(titleXpath, item);
        var href = xpath.select(hrefXpath, item);
        datas.title = title.replace(/(^\s*)|(\s*$)/g, "");
        datas.href = href;
        datasList.push(datas);
    });
    return datasList
}

function parseDetailData(html) {
    var doc = new dom().parseFromString(html);
    // var content = xpath.select('//*[@id="cnblogs_post_body"]', doc);
    var content = xpath.select('string(//div[@class="post"])', doc);
    return content.replace(/(^\s*)|(\s*$)/g, "");
}


async function saveData(conn, pageDatas) {
    //save mongodb
    // util.insertDB(pageDatas)

    // save mysql
    util.insertMysql(conn, pageDatas);
}

// async function saveData(title, conts) {
    // save as file
    // fs.open(`./htmls/${title}.html`, 'w+', function (err, fd) {
    //     if(err) {
    //         console.log(err);
    //         throw err;
    //     }else {
    //         fs.write(fd, conts, 'utf-8', function(err) {
    //             if (err) {
    //                 console.error(err);
    //             }
    //         })
    //     }
    // });
// }


const sleep = (milliseconds) => {
    return new Promise((resolve) => {
        setTimeout(() => resolve(), milliseconds)
    })
}

async function addPageDatas(datasList, num) {
    return new Promise((resolve) => {
        var pageDatas = [];
        datasList.forEach(async function (data) {
            var detailUrl = data.href;
            var title = data.title;
            await sleep(datasList.indexOf(data)*500)
            proxys = await util.getProxy();
            var detailHtml = await parseUrl(detailUrl);
            data.content = parseDetailData(detailHtml);
            pageDatas.push(data)
            if(pageDatas.length == datasList.length) {
                resolve(pageDatas)
            }
            num += 1
            // console.log(num, title);
            logger.info(num, title)
        });
    })
}

async function run(url) {
    const conn = config.connection
    conn.connect();
    var num = 0
    try {
        while (url) {
            proxys = await util.getProxy();
            var listHtml = await parseUrl(url);
            var datasList = await parseListData(listHtml);
            await addPageDatas(datasList, num)
            .then((pageDatas) => {
                saveData(conn, pageDatas);
            })
            .catch((err) => {
                throw new Error(err)
            })
            // 翻页
            var doc = new dom().parseFromString(listHtml);
            var nextUrl = xpath.select('//a[contains(text(),"下一页")]/@href', doc);
            if(nextUrl.length > 0) {
                url = nextUrl[0].nodeValue;
            }else {
                url = '';
            }  
        }
    } catch (error) {
        console.log('run error: ', error.message);
    } finally {
        conn.end();
    }
}


if (require.main === module) {
    var url = 'https://www.cnblogs.com/woodk/';
    run(url)
}





